% parse.pro

:- module(parse,
      [
        parse_term/2,                   % +UntypedTerm, -TypedTerm

        list_of/4,                      % +Type,-List,+2
        set_of/4,                       % +Type,-List,+2
        abstraction_set_of/4,           % +Type,-Term,+2

%        abstraction_from_prolog_list/5, % +Type1,+Type2,-List,+2
        list_of_tokens/4,               % +Type,-List,+2
        set_of_tokens/4,                % +Type,-List,+2
        abstraction_set_of_tokens/4,    % +Type,-Term,+2
        string_of_tokens/4,             % +Type,-Atom,+2

        token/3,                        % -Token,+2
        tokens/3,                       % -Tokens,+2

        tokenise/2,                     % +String,-Tokens
        remove_punctuation/2            % +Xs,-Ys
      ]).


% consult configuration file
:- (getenv('KERNEL_CONFIG',C) ; C='../config'), [C].


/* example usage...

parse(FileName) :-
    read_file_to_codes(FileName, String, []),
    tokenise(String,Tokens),
    phrase(TypedTerm::individual, Tokens),
    write(TypedTerm).

parse :-
    forall(
        direction(Individual, Class),
        (
            parse_term(Individual, TypedTerm),
            E =.. [Class, TypedTerm::individual],
            assert(E::class)
        )
    ).
*/



%------------------------------------------------------------------------------
% PARSE DATA (FIRST ORDER TERMS) TO CANONICAL TYPED FIRST ORDER TERMS
%------------------------------------------------------------------------------


%   parse_term(+UntypedTerm, -TypedTerm)
%
%   use dataset-specific DCG rules to parse string form of UntypedTerm
%   to a type labelled TypedTerm, where a 'type' is a pair <term>::<typename>
%
parse_term(UntypedTerm, TypedTerm) :-
    with_output_to(codes(String), writeq(UntypedTerm)),
    tokenise(String, Tokens),
%writeq(UntypedTerm),nl,
%writeq(Tokens),nl,
%        trace,
    phrase(TypedTerm::individual, Tokens).



%------------------------------------------------------------------------------
% DCG UTILITY PREDICATES FOR PARSING OF BASIC TERMS
%------------------------------------------------------------------------------

/* work in progress
    see ilpnet2people_tmp.pro for example of how intended to work
    and also to_dcg.pro

abstraction_from_prolog_list(T1,T2,A) --> mapping(T1,T2,L),!, { A =.. [abstraction|L] }.
mapping(T,bool,L) --> set_of(T,L).
mapping(T,int,L)  --> mset_of(T,L).

set_of(_T,[]) --> [].
set_of(T,[X::T-true::bool]) --> [X].
set_of(T,[X::T-true::bool|R]) --> [X], [','], set_of(T,R).

% ::set <-> comma separated list of atoms
mset_of(_T,[]) --> [].
mset_of(T,[X::T-1::int]) --> X::T.
mset_of(T,L) --> X::T, [','], mset_of(T,R), {msetadd(X::T,R,L)}.

% msetadd(+Item,+Mset, -NewMset)
% add an item to multiset
msetadd(X,[], [X-1::int]).
msetadd(X,[X-Count::int|R], [X-NewCount::int|R]) :-
    NewCount is Count + 1.
msetadd(X,[Y-Count::int|R1], [Y-Count::int|R2]) :-
    X \== Y,
    msetadd(X,R1,R2).
*/

% ::list <-> comma separated list of atoms
list_of(_,[]::list) --> [].
list_of(T,#(X::T, []::list)::list) --> X::T.
list_of(T,#(X::T, R)::list) --> X::T, [','], list_of(T,R).

% ::set <-> comma separated list of atoms

set_of(_T,[]) --> [].
set_of(T,[X::T-true::bool]) --> X::T.
set_of(T,[X::T-true::bool|R]) --> X::T, [','], set_of(T,R).


abstraction_set_of(T,A) --> set_of(T,L), { A =.. [abstraction|L] }.

% ::list <-> list of tokens
list_of_tokens(T,L) --> ['\''], list_of_tokens_quoted(T,L), ['\''],!.
list_of_tokens(T,#(X::T, R)::list) --> token(X), list_of_tokens(T,R).
list_of_tokens(_,[]::list) --> [].

%%list_of_tokens_quoted(T,#(X::T, R)::list) --> [X], list_of_tokens_quoted(T,R).
list_of_tokens_quoted(T,#(X::T, R)::list) --> token2(X), list_of_tokens_quoted(T,R).
list_of_tokens_quoted(_,[]::list) --> [].

% ::set <-> list of tokens
set_of_tokens(T,S) --> ['\''], set_of_tokens_quoted(T,S), ['\''],!.
set_of_tokens(T,[X::T-true::bool|R]) --> token(X), set_of_tokens(T,R).
set_of_tokens(_,[]) --> [].

%%set_of_tokens_quoted(T,[X::T-true::bool|R]) --> [X], set_of_tokens_quoted(T,R).
set_of_tokens_quoted(T,[X::T-true::bool|R]) --> token2(X), set_of_tokens_quoted(T,R).
set_of_tokens_quoted(_,[]) --> [].

abstraction_set_of_tokens(T,A) --> set_of_tokens(T,L), { A =.. [abstraction|L] }.

% ::string <-> sequence of tokens
string_of_tokens(T,A::T) --> string_of_tokens2(L), { concat_atom(L,'_',A) }.

string_of_tokens2(S) --> ['\''], tokens_any(S), ['\''],!.
string_of_tokens2(S) --> tokens(S).


% token is any non-comma atom
token(T) --> [T], {T\==','}.

% list of tokens <-> list of non-comma atoms
tokens([T|Ts]) --> token(T),tokens(Ts).
tokens([])     --> [].


% token2 is any non-quote atom
token2(T) --> [T], {T\=='\''}.

% list of tokens <-> list of atoms
tokens_any([T|Ts]) --> token2(T), tokens_any(Ts).
tokens_any([])    --> [].





%------------------------------------------------------------------------------
% TOKENISER
%------------------------------------------------------------------------------


/* =================This is commented out as it is duff and runs out of memory

%   tokenise(+Atom,-Tokens)
%
%   Atom is parsed to a list of tokens
%
tokenise(Atom,Tokens) :-
    atomic(Atom),
    atom_codes(Atom,String),
    phrase(tokens(String),Tokens).


%   tokenise(+String,-Tokens)
%
%   String is parsed to a list of tokens
%
tokenise(String,Tokens) :-
    is_list(String),
    phrase(tokens(String),Tokens).



tokens([T|Ts]) --> white_space,token(T),!,tokens(Ts).
tokens([]) --> white_space.

% white_space is zero or more blanks.
white_space --> [C],{code_type(C,space)},!,white_space.
white_space --> [].

token(T) --> special(L), {atom_codes(T,L)}.
token(T) --> word(W),    {atom_codes(T,W)}.
token(T) --> numeral(N), {number_codes(T,N)}.   %doesn't handle missing leading zero on decimals less than 1

special([C]) --> [C], {code_type(C,punct)}.

word([L|Ls]) --> letter(L),word(Ls).
word([L]) --> letter(L).

letter(C) --> [C], {code_type(C,lower)}.
letter(C) --> [C], {code_type(C,upper)}.
%letter(C) --> [U], {code_type(U,upper), code_type(C,to_lower(U))}. %case convert (is this relevant?!)
letter(0'') --> "'".
letter(0'-) --> "-".
letter(0'-) --> "_".

% accept comma chunked numbers
%numeral([C1,C2,C3|N]) --> ",", digit(C1), digit(C2), digit(C3), numeral(N).
%numeral([C1,C2,C3]) --> ",", digit(C1), digit(C2), digit(C3).
numeral([C|N]) --> digit(C), numeral(N).
numeral([C]) --> digit(C).
%numeral(N) --> decimal_part(N).

%decimal_part([0'.|Rest]) --> ".", digit_string(Rest).   %'

digit_string([D|N]) --> digit(D),digit_string(N).
digit_string([D]) --> digit(D).

digit(C) --> [C], {code_type(C,digit)}.

*/

tokenise(Str, Res) :-
   tokenise(Str, [], [], Res).

tokenise([], Acc1, Acc2, Res) :-
    flush(Acc1, Acc2, Res).

tokenise([0'\\ , 0'' |T], Acc1, Acc2, Res) :-   %'  escaped quote char in an atom
    !,
    % apostrophe inside a quoted atom retains its quoting
    append(Acc1, [0'\\ , 0''], Acc3),                    %'
    tokenise(T, Acc3, Acc2, Res).

tokenise([H|T], Acc1, Acc2, Res) :-
   ( isWSpace(H) ->     flush(Acc1,Acc2,Acc3),
                        tokenise(T, [], Acc3, Res)
   ; isSymbol(H,Sym) -> flush(Acc1,Acc2,Acc3),
%                        append(Acc3, [symbol(Sym)], Acc4),
                        append(Acc3, [Sym], Acc4),
                        tokenise(T, [], Acc4, Res)
   ; otherwise   ->     append(Acc1, [H], Acc3),
                        tokenise(T, Acc3, Acc2, Res)
   ).

flush([],A,A) :- !.
flush(A1, A2, A3) :-
   name(N1, A1),
   append(A2, [N1], A3).


%isWSpace(X) :- X=10; X=32.
isWSpace(C) :- code_type(C,space).

%isSymbol(0',,comma).    %'
%isSymbol(0'.,period).   %'
%isSymbol(0'[,open_square).   %'
%isSymbol(0'],close_square).   %'
%isSymbol(0'(,open_bracket).   %'
%isSymbol(0'),close_bracket).   %'

/* works...
isSymbol(0',,',').    %'
isSymbol(0'.,'.').   %'
isSymbol(0'[,'[').   %'
isSymbol(0'],']').   %'
isSymbol(0'(,'(').   %'
isSymbol(0'),')').   %'
*/
isSymbol(C,A) :- code_type(C,punct), char_code(A,C).


/*
isNotComment(X) :- not(X=comment(_)).
isNotSymbol(X) :- not(X=symbol(_)).

isString([]).
isString([H|T]) :-
   number(H),
   isString(T).
*/

/*
test :-
    phrase(tokens(Ws),"Hello world! 123,456.78"),
    write(Ws).
*/



%------------------------------------------------------------------------------
% DATA "CLEANING"
%------------------------------------------------------------------------------


%   remove_punctuation(+Xs,-Ys)
%
%   Ys is the set Xs without atoms that are punctuation characters
%
remove_punctuation(Xs,Ys) :-
    sublist(nonpunct, Xs, Ys).
nonpunct(A) :-
    atom_codes(A,[C|_]),
    \+ code_type(C,punct).


